{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2d124d22-de73-436b-86cd-9b162b469be8",
   "metadata": {
    "id": "2d124d22-de73-436b-86cd-9b162b469be8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: langchain_community in /opt/conda/lib/python3.11/site-packages (0.0.38)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.11/site-packages (from langchain_community) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/conda/lib/python3.11/site-packages (from langchain_community) (2.0.22)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain_community) (3.9.5)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain_community) (0.6.6)\n",
      "Requirement already satisfied: langchain-core<0.2.0,>=0.1.52 in /opt/conda/lib/python3.11/site-packages (from langchain_community) (0.1.52)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.0 in /opt/conda/lib/python3.11/site-packages (from langchain_community) (0.1.82)\n",
      "Requirement already satisfied: numpy<2,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain_community) (1.26.4)\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/conda/lib/python3.11/site-packages (from langchain_community) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain_community) (8.3.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.9.4)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /home/jovyan/.local/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (3.21.2)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain_community) (0.9.0)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.2.0,>=0.1.52->langchain_community) (1.33)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.2.0,>=0.1.52->langchain_community) (23.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain-core<0.2.0,>=0.1.52->langchain_community) (2.7.2)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/jovyan/.local/lib/python3.11/site-packages (from langsmith<0.2.0,>=0.1.0->langchain_community) (3.10.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain_community) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain_community) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain_community) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain_community) (2023.7.22)\n",
      "Requirement already satisfied: typing-extensions>=4.2.0 in /opt/conda/lib/python3.11/site-packages (from SQLAlchemy<3,>=1.4->langchain_community) (4.8.0)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /opt/conda/lib/python3.11/site-packages (from SQLAlchemy<3,>=1.4->langchain_community) (3.0.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.2.0,>=0.1.52->langchain_community) (2.4)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain-core<0.2.0,>=0.1.52->langchain_community) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.3 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain-core<0.2.0,>=0.1.52->langchain_community) (2.18.3)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /home/jovyan/.local/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain_community) (1.0.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: langchain_experimental in /home/jovyan/.local/lib/python3.11/site-packages (0.0.54)\n",
      "Requirement already satisfied: langchain<0.2.0,>=0.1.12 in /opt/conda/lib/python3.11/site-packages (from langchain_experimental) (0.1.20)\n",
      "Requirement already satisfied: langchain-core<0.2.0,>=0.1.31 in /opt/conda/lib/python3.11/site-packages (from langchain_experimental) (0.1.52)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/conda/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (2.0.22)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (3.9.5)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (0.6.6)\n",
      "Requirement already satisfied: langchain-community<0.1,>=0.0.38 in /opt/conda/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (0.0.38)\n",
      "Requirement already satisfied: langchain-text-splitters<0.1,>=0.0.1 in /opt/conda/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (0.0.2)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /opt/conda/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (0.1.82)\n",
      "Requirement already satisfied: numpy<2,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (1.26.4)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (2.7.2)\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/conda/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain<0.2.0,>=0.1.12->langchain_experimental) (8.3.0)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.2.0,>=0.1.31->langchain_experimental) (1.33)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.2.0,>=0.1.31->langchain_experimental) (23.2)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain<0.2.0,>=0.1.12->langchain_experimental) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain<0.2.0,>=0.1.12->langchain_experimental) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain<0.2.0,>=0.1.12->langchain_experimental) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain<0.2.0,>=0.1.12->langchain_experimental) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain<0.2.0,>=0.1.12->langchain_experimental) (1.9.4)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /home/jovyan/.local/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain<0.2.0,>=0.1.12->langchain_experimental) (3.21.2)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain<0.2.0,>=0.1.12->langchain_experimental) (0.9.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.2.0,>=0.1.31->langchain_experimental) (2.4)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/jovyan/.local/lib/python3.11/site-packages (from langsmith<0.2.0,>=0.1.17->langchain<0.2.0,>=0.1.12->langchain_experimental) (3.10.3)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain<0.2.0,>=0.1.12->langchain_experimental) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.3 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain<0.2.0,>=0.1.12->langchain_experimental) (2.18.3)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1->langchain<0.2.0,>=0.1.12->langchain_experimental) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain<0.2.0,>=0.1.12->langchain_experimental) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain<0.2.0,>=0.1.12->langchain_experimental) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain<0.2.0,>=0.1.12->langchain_experimental) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain<0.2.0,>=0.1.12->langchain_experimental) (2023.7.22)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /opt/conda/lib/python3.11/site-packages (from SQLAlchemy<3,>=1.4->langchain<0.2.0,>=0.1.12->langchain_experimental) (3.0.0)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /home/jovyan/.local/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain<0.2.0,>=0.1.12->langchain_experimental) (1.0.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: langchain-openai in /opt/conda/lib/python3.11/site-packages (0.1.7)\n",
      "Requirement already satisfied: langchain-core<0.3,>=0.1.46 in /opt/conda/lib/python3.11/site-packages (from langchain-openai) (0.1.52)\n",
      "Requirement already satisfied: openai<2.0.0,>=1.24.0 in /opt/conda/lib/python3.11/site-packages (from langchain-openai) (1.35.7)\n",
      "Requirement already satisfied: tiktoken<1,>=0.7 in /opt/conda/lib/python3.11/site-packages (from langchain-openai) (0.7.0)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.46->langchain-openai) (6.0.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.46->langchain-openai) (1.33)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.0 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.46->langchain-openai) (0.1.82)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.46->langchain-openai) (23.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.46->langchain-openai) (2.7.2)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.46->langchain-openai) (8.3.0)\n",
      "Requirement already satisfied: anyio<5,>=3.5.0 in /home/jovyan/.local/lib/python3.11/site-packages (from openai<2.0.0,>=1.24.0->langchain-openai) (3.7.1)\n",
      "Requirement already satisfied: distro<2,>=1.7.0 in /home/jovyan/.local/lib/python3.11/site-packages (from openai<2.0.0,>=1.24.0->langchain-openai) (1.9.0)\n",
      "Requirement already satisfied: httpx<1,>=0.23.0 in /home/jovyan/.local/lib/python3.11/site-packages (from openai<2.0.0,>=1.24.0->langchain-openai) (0.27.0)\n",
      "Requirement already satisfied: sniffio in /opt/conda/lib/python3.11/site-packages (from openai<2.0.0,>=1.24.0->langchain-openai) (1.3.0)\n",
      "Requirement already satisfied: tqdm>4 in /opt/conda/lib/python3.11/site-packages (from openai<2.0.0,>=1.24.0->langchain-openai) (4.66.1)\n",
      "Requirement already satisfied: typing-extensions<5,>=4.7 in /opt/conda/lib/python3.11/site-packages (from openai<2.0.0,>=1.24.0->langchain-openai) (4.8.0)\n",
      "Requirement already satisfied: regex>=2022.1.18 in /opt/conda/lib/python3.11/site-packages (from tiktoken<1,>=0.7->langchain-openai) (2024.5.15)\n",
      "Requirement already satisfied: requests>=2.26.0 in /opt/conda/lib/python3.11/site-packages (from tiktoken<1,>=0.7->langchain-openai) (2.31.0)\n",
      "Requirement already satisfied: idna>=2.8 in /opt/conda/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.24.0->langchain-openai) (3.4)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.24.0->langchain-openai) (2023.7.22)\n",
      "Requirement already satisfied: httpcore==1.* in /home/jovyan/.local/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai<2.0.0,>=1.24.0->langchain-openai) (1.0.5)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/jovyan/.local/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai<2.0.0,>=1.24.0->langchain-openai) (0.14.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3,>=0.1.46->langchain-openai) (2.4)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/jovyan/.local/lib/python3.11/site-packages (from langsmith<0.2.0,>=0.1.0->langchain-core<0.3,>=0.1.46->langchain-openai) (3.10.3)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain-core<0.3,>=0.1.46->langchain-openai) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.3 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain-core<0.3,>=0.1.46->langchain-openai) (2.18.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken<1,>=0.7->langchain-openai) (3.3.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests>=2.26.0->tiktoken<1,>=0.7->langchain-openai) (2.0.7)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: langchainhub in /opt/conda/lib/python3.11/site-packages (0.1.20)\n",
      "Requirement already satisfied: packaging<25,>=23.2 in /opt/conda/lib/python3.11/site-packages (from langchainhub) (23.2)\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/conda/lib/python3.11/site-packages (from langchainhub) (2.31.0)\n",
      "Requirement already satisfied: types-requests<3.0.0.0,>=2.31.0.2 in /home/jovyan/.local/lib/python3.11/site-packages (from langchainhub) (2.31.0.20240406)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchainhub) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchainhub) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchainhub) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchainhub) (2023.7.22)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: chromadb in /opt/conda/lib/python3.11/site-packages (0.5.3)\n",
      "Requirement already satisfied: build>=1.0.3 in /opt/conda/lib/python3.11/site-packages (from chromadb) (1.2.1)\n",
      "Requirement already satisfied: requests>=2.28 in /opt/conda/lib/python3.11/site-packages (from chromadb) (2.31.0)\n",
      "Requirement already satisfied: pydantic>=1.9 in /home/jovyan/.local/lib/python3.11/site-packages (from chromadb) (2.7.2)\n",
      "Requirement already satisfied: chroma-hnswlib==0.7.3 in /opt/conda/lib/python3.11/site-packages (from chromadb) (0.7.3)\n",
      "Requirement already satisfied: fastapi>=0.95.2 in /opt/conda/lib/python3.11/site-packages (from chromadb) (0.111.0)\n",
      "Requirement already satisfied: uvicorn>=0.18.3 in /opt/conda/lib/python3.11/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.30.1)\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.22.5 in /home/jovyan/.local/lib/python3.11/site-packages (from chromadb) (1.26.4)\n",
      "Requirement already satisfied: posthog>=2.4.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (3.5.0)\n",
      "Requirement already satisfied: typing-extensions>=4.5.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (4.8.0)\n",
      "Requirement already satisfied: onnxruntime>=1.14.1 in /opt/conda/lib/python3.11/site-packages (from chromadb) (1.18.1)\n",
      "Requirement already satisfied: opentelemetry-api>=1.2.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (1.25.0)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (1.25.0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (0.46b0)\n",
      "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (1.25.0)\n",
      "Requirement already satisfied: tokenizers>=0.13.2 in /home/jovyan/.local/lib/python3.11/site-packages (from chromadb) (0.19.1)\n",
      "Requirement already satisfied: pypika>=0.48.9 in /opt/conda/lib/python3.11/site-packages (from chromadb) (0.48.9)\n",
      "Requirement already satisfied: tqdm>=4.65.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (4.66.1)\n",
      "Requirement already satisfied: overrides>=7.3.1 in /opt/conda/lib/python3.11/site-packages (from chromadb) (7.4.0)\n",
      "Requirement already satisfied: importlib-resources in /opt/conda/lib/python3.11/site-packages (from chromadb) (6.1.0)\n",
      "Requirement already satisfied: grpcio>=1.58.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (1.64.1)\n",
      "Requirement already satisfied: bcrypt>=4.0.1 in /opt/conda/lib/python3.11/site-packages (from chromadb) (4.1.3)\n",
      "Requirement already satisfied: typer>=0.9.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (0.12.3)\n",
      "Requirement already satisfied: kubernetes>=28.1.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (30.1.0)\n",
      "Requirement already satisfied: tenacity>=8.2.3 in /home/jovyan/.local/lib/python3.11/site-packages (from chromadb) (8.3.0)\n",
      "Requirement already satisfied: PyYAML>=6.0.0 in /opt/conda/lib/python3.11/site-packages (from chromadb) (6.0.1)\n",
      "Requirement already satisfied: mmh3>=4.0.1 in /opt/conda/lib/python3.11/site-packages (from chromadb) (4.1.0)\n",
      "Requirement already satisfied: orjson>=3.9.12 in /home/jovyan/.local/lib/python3.11/site-packages (from chromadb) (3.10.3)\n",
      "Requirement already satisfied: httpx>=0.27.0 in /home/jovyan/.local/lib/python3.11/site-packages (from chromadb) (0.27.0)\n",
      "Requirement already satisfied: packaging>=19.1 in /opt/conda/lib/python3.11/site-packages (from build>=1.0.3->chromadb) (23.2)\n",
      "Requirement already satisfied: pyproject_hooks in /opt/conda/lib/python3.11/site-packages (from build>=1.0.3->chromadb) (1.1.0)\n",
      "Requirement already satisfied: starlette<0.38.0,>=0.37.2 in /opt/conda/lib/python3.11/site-packages (from fastapi>=0.95.2->chromadb) (0.37.2)\n",
      "Requirement already satisfied: fastapi-cli>=0.0.2 in /opt/conda/lib/python3.11/site-packages (from fastapi>=0.95.2->chromadb) (0.0.4)\n",
      "Requirement already satisfied: jinja2>=2.11.2 in /opt/conda/lib/python3.11/site-packages (from fastapi>=0.95.2->chromadb) (3.1.2)\n",
      "Requirement already satisfied: python-multipart>=0.0.7 in /home/jovyan/.local/lib/python3.11/site-packages (from fastapi>=0.95.2->chromadb) (0.0.9)\n",
      "Requirement already satisfied: ujson!=4.0.2,!=4.1.0,!=4.2.0,!=4.3.0,!=5.0.0,!=5.1.0,>=4.0.1 in /opt/conda/lib/python3.11/site-packages (from fastapi>=0.95.2->chromadb) (5.10.0)\n",
      "Requirement already satisfied: email_validator>=2.0.0 in /opt/conda/lib/python3.11/site-packages (from fastapi>=0.95.2->chromadb) (2.2.0)\n",
      "Requirement already satisfied: anyio in /home/jovyan/.local/lib/python3.11/site-packages (from httpx>=0.27.0->chromadb) (3.7.1)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.11/site-packages (from httpx>=0.27.0->chromadb) (2023.7.22)\n",
      "Requirement already satisfied: httpcore==1.* in /home/jovyan/.local/lib/python3.11/site-packages (from httpx>=0.27.0->chromadb) (1.0.5)\n",
      "Requirement already satisfied: idna in /opt/conda/lib/python3.11/site-packages (from httpx>=0.27.0->chromadb) (3.4)\n",
      "Requirement already satisfied: sniffio in /opt/conda/lib/python3.11/site-packages (from httpx>=0.27.0->chromadb) (1.3.0)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/jovyan/.local/lib/python3.11/site-packages (from httpcore==1.*->httpx>=0.27.0->chromadb) (0.14.0)\n",
      "Requirement already satisfied: six>=1.9.0 in /opt/conda/lib/python3.11/site-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: python-dateutil>=2.5.3 in /opt/conda/lib/python3.11/site-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n",
      "Requirement already satisfied: google-auth>=1.0.1 in /opt/conda/lib/python3.11/site-packages (from kubernetes>=28.1.0->chromadb) (2.30.0)\n",
      "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /opt/conda/lib/python3.11/site-packages (from kubernetes>=28.1.0->chromadb) (1.6.4)\n",
      "Requirement already satisfied: requests-oauthlib in /opt/conda/lib/python3.11/site-packages (from kubernetes>=28.1.0->chromadb) (2.0.0)\n",
      "Requirement already satisfied: oauthlib>=3.2.2 in /opt/conda/lib/python3.11/site-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n",
      "Requirement already satisfied: urllib3>=1.24.2 in /opt/conda/lib/python3.11/site-packages (from kubernetes>=28.1.0->chromadb) (2.0.7)\n",
      "Requirement already satisfied: coloredlogs in /opt/conda/lib/python3.11/site-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n",
      "Requirement already satisfied: flatbuffers in /opt/conda/lib/python3.11/site-packages (from onnxruntime>=1.14.1->chromadb) (24.3.25)\n",
      "Requirement already satisfied: protobuf in /opt/conda/lib/python3.11/site-packages (from onnxruntime>=1.14.1->chromadb) (4.25.3)\n",
      "Requirement already satisfied: sympy in /opt/conda/lib/python3.11/site-packages (from onnxruntime>=1.14.1->chromadb) (1.12.1)\n",
      "Requirement already satisfied: deprecated>=1.2.6 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.14)\n",
      "Requirement already satisfied: importlib-metadata<=7.1,>=6.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-api>=1.2.0->chromadb) (6.8.0)\n",
      "Requirement already satisfied: googleapis-common-protos~=1.52 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.63.2)\n",
      "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.25.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.25.0)\n",
      "Requirement already satisfied: opentelemetry-proto==1.25.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.25.0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.46b0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.46b0)\n",
      "Requirement already satisfied: opentelemetry-instrumentation==0.46b0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.46b0)\n",
      "Requirement already satisfied: opentelemetry-semantic-conventions==0.46b0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.46b0)\n",
      "Requirement already satisfied: opentelemetry-util-http==0.46b0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.46b0)\n",
      "Requirement already satisfied: setuptools>=16.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (68.2.2)\n",
      "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.16.0)\n",
      "Requirement already satisfied: asgiref~=3.0 in /opt/conda/lib/python3.11/site-packages (from opentelemetry-instrumentation-asgi==0.46b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (3.8.1)\n",
      "Requirement already satisfied: monotonic>=1.5 in /opt/conda/lib/python3.11/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n",
      "Requirement already satisfied: backoff>=1.10.0 in /opt/conda/lib/python3.11/site-packages (from posthog>=2.4.0->chromadb) (2.2.1)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic>=1.9->chromadb) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.3 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic>=1.9->chromadb) (2.18.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests>=2.28->chromadb) (3.3.0)\n",
      "Requirement already satisfied: huggingface-hub<1.0,>=0.16.4 in /opt/conda/lib/python3.11/site-packages (from tokenizers>=0.13.2->chromadb) (0.23.4)\n",
      "Requirement already satisfied: click>=8.0.0 in /opt/conda/lib/python3.11/site-packages (from typer>=0.9.0->chromadb) (8.1.7)\n",
      "Requirement already satisfied: shellingham>=1.3.0 in /opt/conda/lib/python3.11/site-packages (from typer>=0.9.0->chromadb) (1.5.4)\n",
      "Requirement already satisfied: rich>=10.11.0 in /opt/conda/lib/python3.11/site-packages (from typer>=0.9.0->chromadb) (13.7.1)\n",
      "Requirement already satisfied: httptools>=0.5.0 in /opt/conda/lib/python3.11/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.1)\n",
      "Requirement already satisfied: python-dotenv>=0.13 in /home/jovyan/.local/lib/python3.11/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.1)\n",
      "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /opt/conda/lib/python3.11/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.19.0)\n",
      "Requirement already satisfied: watchfiles>=0.13 in /opt/conda/lib/python3.11/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.22.0)\n",
      "Requirement already satisfied: websockets>=10.4 in /home/jovyan/.local/lib/python3.11/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (11.0.3)\n",
      "Requirement already satisfied: dnspython>=2.0.0 in /opt/conda/lib/python3.11/site-packages (from email_validator>=2.0.0->fastapi>=0.95.2->chromadb) (2.6.1)\n",
      "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /opt/conda/lib/python3.11/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.3)\n",
      "Requirement already satisfied: pyasn1-modules>=0.2.1 in /opt/conda/lib/python3.11/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.4.0)\n",
      "Requirement already satisfied: rsa<5,>=3.1.4 in /opt/conda/lib/python3.11/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n",
      "Requirement already satisfied: filelock in /opt/conda/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.15.4)\n",
      "Requirement already satisfied: fsspec>=2023.5.0 in /home/jovyan/.local/lib/python3.11/site-packages (from huggingface-hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (2024.3.1)\n",
      "Requirement already satisfied: zipp>=0.5 in /opt/conda/lib/python3.11/site-packages (from importlib-metadata<=7.1,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n",
      "Requirement already satisfied: MarkupSafe>=2.0 in /opt/conda/lib/python3.11/site-packages (from jinja2>=2.11.2->fastapi>=0.95.2->chromadb) (2.1.3)\n",
      "Requirement already satisfied: markdown-it-py>=2.2.0 in /opt/conda/lib/python3.11/site-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (3.0.0)\n",
      "Requirement already satisfied: pygments<3.0.0,>=2.13.0 in /opt/conda/lib/python3.11/site-packages (from rich>=10.11.0->typer>=0.9.0->chromadb) (2.16.1)\n",
      "Requirement already satisfied: humanfriendly>=9.1 in /opt/conda/lib/python3.11/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n",
      "Requirement already satisfied: mpmath<1.4.0,>=1.1.0 in /opt/conda/lib/python3.11/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n",
      "Requirement already satisfied: mdurl~=0.1 in /opt/conda/lib/python3.11/site-packages (from markdown-it-py>=2.2.0->rich>=10.11.0->typer>=0.9.0->chromadb) (0.1.2)\n",
      "Requirement already satisfied: pyasn1<0.7.0,>=0.4.6 in /opt/conda/lib/python3.11/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.6.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: langchain in /opt/conda/lib/python3.11/site-packages (0.1.20)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.11/site-packages (from langchain) (6.0.1)\n",
      "Requirement already satisfied: SQLAlchemy<3,>=1.4 in /opt/conda/lib/python3.11/site-packages (from langchain) (2.0.22)\n",
      "Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain) (3.9.5)\n",
      "Requirement already satisfied: dataclasses-json<0.7,>=0.5.7 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain) (0.6.6)\n",
      "Requirement already satisfied: langchain-community<0.1,>=0.0.38 in /opt/conda/lib/python3.11/site-packages (from langchain) (0.0.38)\n",
      "Requirement already satisfied: langchain-core<0.2.0,>=0.1.52 in /opt/conda/lib/python3.11/site-packages (from langchain) (0.1.52)\n",
      "Requirement already satisfied: langchain-text-splitters<0.1,>=0.0.1 in /opt/conda/lib/python3.11/site-packages (from langchain) (0.0.2)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.17 in /opt/conda/lib/python3.11/site-packages (from langchain) (0.1.82)\n",
      "Requirement already satisfied: numpy<2,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain) (1.26.4)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain) (2.7.2)\n",
      "Requirement already satisfied: requests<3,>=2 in /opt/conda/lib/python3.11/site-packages (from langchain) (2.31.0)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain) (8.3.0)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.3.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /opt/conda/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.4.1)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (6.0.5)\n",
      "Requirement already satisfied: yarl<2.0,>=1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from aiohttp<4.0.0,>=3.8.3->langchain) (1.9.4)\n",
      "Requirement already satisfied: marshmallow<4.0.0,>=3.18.0 in /home/jovyan/.local/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (3.21.2)\n",
      "Requirement already satisfied: typing-inspect<1,>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from dataclasses-json<0.7,>=0.5.7->langchain) (0.9.0)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.2.0,>=0.1.52->langchain) (1.33)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.2.0,>=0.1.52->langchain) (23.2)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/jovyan/.local/lib/python3.11/site-packages (from langsmith<0.2.0,>=0.1.17->langchain) (3.10.3)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.3 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (2.18.3)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1->langchain) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain) (3.3.0)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /opt/conda/lib/python3.11/site-packages (from requests<3,>=2->langchain) (2023.7.22)\n",
      "Requirement already satisfied: greenlet!=0.4.17 in /opt/conda/lib/python3.11/site-packages (from SQLAlchemy<3,>=1.4->langchain) (3.0.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.2.0,>=0.1.52->langchain) (2.4)\n",
      "Requirement already satisfied: mypy-extensions>=0.3.0 in /home/jovyan/.local/lib/python3.11/site-packages (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain) (1.0.0)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: python-dotenv in /home/jovyan/.local/lib/python3.11/site-packages (1.0.1)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Requirement already satisfied: rank_bm25 in /opt/conda/lib/python3.11/site-packages (0.2.2)\n",
      "Requirement already satisfied: numpy in /home/jovyan/.local/lib/python3.11/site-packages (from rank_bm25) (1.26.4)\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Collecting faiss-cpu\n",
      "  Downloading faiss_cpu-1.8.0.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl.metadata (3.7 kB)\n",
      "Requirement already satisfied: numpy<2.0,>=1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from faiss-cpu) (1.26.4)\n",
      "Requirement already satisfied: packaging in /opt/conda/lib/python3.11/site-packages (from faiss-cpu) (23.2)\n",
      "Downloading faiss_cpu-1.8.0.post1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl (3.6 MB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.6/3.6 MB\u001b[0m \u001b[31m4.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m00:01\u001b[0m00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: faiss-cpu\n",
      "Successfully installed faiss-cpu-1.8.0.post1\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Collecting weaviate-client\n",
      "  Downloading weaviate_client-4.6.5-py3-none-any.whl.metadata (3.3 kB)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.30.0 in /opt/conda/lib/python3.11/site-packages (from weaviate-client) (2.31.0)\n",
      "Requirement already satisfied: httpx<=0.27.0,>=0.25.0 in /home/jovyan/.local/lib/python3.11/site-packages (from weaviate-client) (0.27.0)\n",
      "Collecting validators==0.28.3 (from weaviate-client)\n",
      "  Using cached validators-0.28.3-py3-none-any.whl.metadata (3.6 kB)\n",
      "Collecting authlib<2.0.0,>=1.2.1 (from weaviate-client)\n",
      "  Using cached Authlib-1.3.1-py2.py3-none-any.whl.metadata (3.8 kB)\n",
      "Requirement already satisfied: pydantic<3.0.0,>=2.5.0 in /home/jovyan/.local/lib/python3.11/site-packages (from weaviate-client) (2.7.2)\n",
      "Requirement already satisfied: grpcio<2.0.0,>=1.57.0 in /opt/conda/lib/python3.11/site-packages (from weaviate-client) (1.64.1)\n",
      "Collecting grpcio-tools<2.0.0,>=1.57.0 (from weaviate-client)\n",
      "  Using cached grpcio_tools-1.64.1-cp311-cp311-manylinux_2_17_aarch64.whl.metadata (5.3 kB)\n",
      "Collecting grpcio-health-checking<2.0.0,>=1.57.0 (from weaviate-client)\n",
      "  Using cached grpcio_health_checking-1.64.1-py3-none-any.whl.metadata (1.1 kB)\n",
      "Requirement already satisfied: cryptography in /opt/conda/lib/python3.11/site-packages (from authlib<2.0.0,>=1.2.1->weaviate-client) (41.0.4)\n",
      "Collecting protobuf<6.0dev,>=5.26.1 (from grpcio-health-checking<2.0.0,>=1.57.0->weaviate-client)\n",
      "  Using cached protobuf-5.27.2-cp38-abi3-manylinux2014_aarch64.whl.metadata (592 bytes)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.11/site-packages (from grpcio-tools<2.0.0,>=1.57.0->weaviate-client) (68.2.2)\n",
      "Requirement already satisfied: anyio in /home/jovyan/.local/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client) (3.7.1)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client) (2023.7.22)\n",
      "Requirement already satisfied: httpcore==1.* in /home/jovyan/.local/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client) (1.0.5)\n",
      "Requirement already satisfied: idna in /opt/conda/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client) (3.4)\n",
      "Requirement already satisfied: sniffio in /opt/conda/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client) (1.3.0)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/jovyan/.local/lib/python3.11/site-packages (from httpcore==1.*->httpx<=0.27.0,>=0.25.0->weaviate-client) (0.14.0)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.5.0->weaviate-client) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.3 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.5.0->weaviate-client) (2.18.3)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /opt/conda/lib/python3.11/site-packages (from pydantic<3.0.0,>=2.5.0->weaviate-client) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests<3.0.0,>=2.30.0->weaviate-client) (3.3.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests<3.0.0,>=2.30.0->weaviate-client) (2.0.7)\n",
      "Requirement already satisfied: cffi>=1.12 in /opt/conda/lib/python3.11/site-packages (from cryptography->authlib<2.0.0,>=1.2.1->weaviate-client) (1.16.0)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.11/site-packages (from cffi>=1.12->cryptography->authlib<2.0.0,>=1.2.1->weaviate-client) (2.21)\n",
      "Downloading weaviate_client-4.6.5-py3-none-any.whl (325 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m325.7/325.7 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hUsing cached validators-0.28.3-py3-none-any.whl (40 kB)\n",
      "Using cached Authlib-1.3.1-py2.py3-none-any.whl (223 kB)\n",
      "Using cached grpcio_health_checking-1.64.1-py3-none-any.whl (18 kB)\n",
      "Using cached grpcio_tools-1.64.1-cp311-cp311-manylinux_2_17_aarch64.whl (2.2 MB)\n",
      "Downloading protobuf-5.27.2-cp38-abi3-manylinux2014_aarch64.whl (307 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m307.1/307.1 kB\u001b[0m \u001b[31m3.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: validators, protobuf, grpcio-tools, grpcio-health-checking, authlib, weaviate-client\n",
      "  Attempting uninstall: protobuf\n",
      "    Found existing installation: protobuf 4.25.3\n",
      "    Uninstalling protobuf-4.25.3:\n",
      "      Successfully uninstalled protobuf-4.25.3\n",
      "\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n",
      "streamlit 1.33.0 requires protobuf<5,>=3.20, but you have protobuf 5.27.2 which is incompatible.\n",
      "opentelemetry-proto 1.25.0 requires protobuf<5.0,>=3.19, but you have protobuf 5.27.2 which is incompatible.\u001b[0m\u001b[31m\n",
      "\u001b[0mSuccessfully installed authlib-1.3.1 grpcio-health-checking-1.64.1 grpcio-tools-1.64.1 protobuf-5.27.2 validators-0.28.3 weaviate-client-4.6.5\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n",
      "Collecting langchain-weaviate\n",
      "  Using cached langchain_weaviate-0.0.2-py3-none-any.whl.metadata (2.7 kB)\n",
      "Requirement already satisfied: langchain-core<0.3,>=0.1.33 in /opt/conda/lib/python3.11/site-packages (from langchain-weaviate) (0.1.52)\n",
      "Requirement already satisfied: numpy<2.0.0,>=1.26.2 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain-weaviate) (1.26.4)\n",
      "Collecting simsimd<5.0.0,>=3.6.1 (from langchain-weaviate)\n",
      "  Downloading simsimd-4.4.0-cp311-cp311-manylinux_2_28_aarch64.whl.metadata (31 kB)\n",
      "Requirement already satisfied: weaviate-client<5.0.0,>=4.0.0 in /opt/conda/lib/python3.11/site-packages (from langchain-weaviate) (4.6.5)\n",
      "Requirement already satisfied: PyYAML>=5.3 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.33->langchain-weaviate) (6.0.1)\n",
      "Requirement already satisfied: jsonpatch<2.0,>=1.33 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.33->langchain-weaviate) (1.33)\n",
      "Requirement already satisfied: langsmith<0.2.0,>=0.1.0 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.33->langchain-weaviate) (0.1.82)\n",
      "Requirement already satisfied: packaging<24.0,>=23.2 in /opt/conda/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.33->langchain-weaviate) (23.2)\n",
      "Requirement already satisfied: pydantic<3,>=1 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.33->langchain-weaviate) (2.7.2)\n",
      "Requirement already satisfied: tenacity<9.0.0,>=8.1.0 in /home/jovyan/.local/lib/python3.11/site-packages (from langchain-core<0.3,>=0.1.33->langchain-weaviate) (8.3.0)\n",
      "Requirement already satisfied: requests<3.0.0,>=2.30.0 in /opt/conda/lib/python3.11/site-packages (from weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (2.31.0)\n",
      "Requirement already satisfied: httpx<=0.27.0,>=0.25.0 in /home/jovyan/.local/lib/python3.11/site-packages (from weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (0.27.0)\n",
      "Requirement already satisfied: validators==0.28.3 in /opt/conda/lib/python3.11/site-packages (from weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (0.28.3)\n",
      "Requirement already satisfied: authlib<2.0.0,>=1.2.1 in /opt/conda/lib/python3.11/site-packages (from weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (1.3.1)\n",
      "Requirement already satisfied: grpcio<2.0.0,>=1.57.0 in /opt/conda/lib/python3.11/site-packages (from weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (1.64.1)\n",
      "Requirement already satisfied: grpcio-tools<2.0.0,>=1.57.0 in /opt/conda/lib/python3.11/site-packages (from weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (1.64.1)\n",
      "Requirement already satisfied: grpcio-health-checking<2.0.0,>=1.57.0 in /opt/conda/lib/python3.11/site-packages (from weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (1.64.1)\n",
      "Requirement already satisfied: cryptography in /opt/conda/lib/python3.11/site-packages (from authlib<2.0.0,>=1.2.1->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (41.0.4)\n",
      "Requirement already satisfied: protobuf<6.0dev,>=5.26.1 in /opt/conda/lib/python3.11/site-packages (from grpcio-health-checking<2.0.0,>=1.57.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (5.27.2)\n",
      "Requirement already satisfied: setuptools in /opt/conda/lib/python3.11/site-packages (from grpcio-tools<2.0.0,>=1.57.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (68.2.2)\n",
      "Requirement already satisfied: anyio in /home/jovyan/.local/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (3.7.1)\n",
      "Requirement already satisfied: certifi in /opt/conda/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (2023.7.22)\n",
      "Requirement already satisfied: httpcore==1.* in /home/jovyan/.local/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (1.0.5)\n",
      "Requirement already satisfied: idna in /opt/conda/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (3.4)\n",
      "Requirement already satisfied: sniffio in /opt/conda/lib/python3.11/site-packages (from httpx<=0.27.0,>=0.25.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (1.3.0)\n",
      "Requirement already satisfied: h11<0.15,>=0.13 in /home/jovyan/.local/lib/python3.11/site-packages (from httpcore==1.*->httpx<=0.27.0,>=0.25.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (0.14.0)\n",
      "Requirement already satisfied: jsonpointer>=1.9 in /opt/conda/lib/python3.11/site-packages (from jsonpatch<2.0,>=1.33->langchain-core<0.3,>=0.1.33->langchain-weaviate) (2.4)\n",
      "Requirement already satisfied: orjson<4.0.0,>=3.9.14 in /home/jovyan/.local/lib/python3.11/site-packages (from langsmith<0.2.0,>=0.1.0->langchain-core<0.3,>=0.1.33->langchain-weaviate) (3.10.3)\n",
      "Requirement already satisfied: annotated-types>=0.4.0 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain-core<0.3,>=0.1.33->langchain-weaviate) (0.6.0)\n",
      "Requirement already satisfied: pydantic-core==2.18.3 in /home/jovyan/.local/lib/python3.11/site-packages (from pydantic<3,>=1->langchain-core<0.3,>=0.1.33->langchain-weaviate) (2.18.3)\n",
      "Requirement already satisfied: typing-extensions>=4.6.1 in /opt/conda/lib/python3.11/site-packages (from pydantic<3,>=1->langchain-core<0.3,>=0.1.33->langchain-weaviate) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /opt/conda/lib/python3.11/site-packages (from requests<3.0.0,>=2.30.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (3.3.0)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /opt/conda/lib/python3.11/site-packages (from requests<3.0.0,>=2.30.0->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (2.0.7)\n",
      "Requirement already satisfied: cffi>=1.12 in /opt/conda/lib/python3.11/site-packages (from cryptography->authlib<2.0.0,>=1.2.1->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (1.16.0)\n",
      "Requirement already satisfied: pycparser in /opt/conda/lib/python3.11/site-packages (from cffi>=1.12->cryptography->authlib<2.0.0,>=1.2.1->weaviate-client<5.0.0,>=4.0.0->langchain-weaviate) (2.21)\n",
      "Using cached langchain_weaviate-0.0.2-py3-none-any.whl (10 kB)\n",
      "Downloading simsimd-4.4.0-cp311-cp311-manylinux_2_28_aarch64.whl (258 kB)\n",
      "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m258.7/258.7 kB\u001b[0m \u001b[31m3.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0ma \u001b[36m0:00:01\u001b[0m\n",
      "\u001b[?25hInstalling collected packages: simsimd, langchain-weaviate\n",
      "Successfully installed langchain-weaviate-0.0.2 simsimd-4.4.0\n",
      "\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m24.0\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.1.1\u001b[0m\n",
      "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install langchain_community\n",
    "%pip install langchain_experimental\n",
    "%pip install langchain-openai\n",
    "%pip install langchainhub\n",
    "%pip install chromadb\n",
    "%pip install langchain\n",
    "%pip install python-dotenv\n",
    "%pip install PyPDF2 -q --user\n",
    "%pip install rank_bm25\n",
    "\n",
    "# new vector stores\n",
    "%pip install faiss-cpu\n",
    "%pip install weaviate-client\n",
    "%pip install langchain-weaviate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f884314f-870c-4bfb-b6c1-a5b4801ec172",
   "metadata": {
    "executionInfo": {
     "elapsed": 7390,
     "status": "ok",
     "timestamp": 1715455737041,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "f884314f-870c-4bfb-b6c1-a5b4801ec172"
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "from langchain_openai import ChatOpenAI, OpenAIEmbeddings\n",
    "from langchain import hub\n",
    "from langchain_core.output_parsers import StrOutputParser\n",
    "from langchain_core.runnables import RunnablePassthrough\n",
    "import chromadb\n",
    "from langchain_community.vectorstores import Chroma\n",
    "from langchain_core.runnables import RunnableParallel\n",
    "from dotenv import load_dotenv, find_dotenv\n",
    "from langchain_core.prompts import PromptTemplate\n",
    "from PyPDF2 import PdfReader\n",
    "from langchain.text_splitter import RecursiveCharacterTextSplitter\n",
    "from langchain_core.documents.base import Document\n",
    "from langchain_community.retrievers import BM25Retriever\n",
    "from langchain.retrievers import EnsembleRetriever"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "eba3468a-d7c2-4a79-8df2-c335542950f2",
   "metadata": {
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1715455737042,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "eba3468a-d7c2-4a79-8df2-c335542950f2"
   },
   "outputs": [],
   "source": [
    "# variables\n",
    "_ = load_dotenv(dotenv_path='env.txt')\n",
    "os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY')\n",
    "openai.api_key = os.environ['OPENAI_API_KEY']\n",
    "embedding_function = OpenAIEmbeddings()\n",
    "llm = ChatOpenAI(model_name=\"gpt-4o-mini\")\n",
    "pdf_path = \"google-2023-environmental-report.pdf\"\n",
    "collection_name = \"google_environmental_report\"\n",
    "str_output_parser = StrOutputParser()\n",
    "user_query = \"What are Google's environmental initiatives?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d3ad428a-3eb6-40ec-a1a5-62565ead1e5b",
   "metadata": {
    "id": "d3ad428a-3eb6-40ec-a1a5-62565ead1e5b"
   },
   "outputs": [],
   "source": [
    "#### INDEXING ####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "98ccda2c-0f4c-41c5-804d-2227cdf35aa7",
   "metadata": {
    "executionInfo": {
     "elapsed": 19512,
     "status": "ok",
     "timestamp": 1715455756551,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "98ccda2c-0f4c-41c5-804d-2227cdf35aa7"
   },
   "outputs": [],
   "source": [
    "# Load the PDF and extract text\n",
    "pdf_reader = PdfReader(pdf_path)\n",
    "text = \"\"\n",
    "for page in pdf_reader.pages:\n",
    "    text += page.extract_text()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "927a4c65-aa05-486c-8295-2f99673e7c20",
   "metadata": {
    "executionInfo": {
     "elapsed": 12,
     "status": "ok",
     "timestamp": 1715455756552,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "927a4c65-aa05-486c-8295-2f99673e7c20"
   },
   "outputs": [],
   "source": [
    "# Split\n",
    "character_splitter = RecursiveCharacterTextSplitter(\n",
    "    separators=[\"\\n\\n\", \"\\n\", \". \", \" \", \"\"],\n",
    "    chunk_size=1000,\n",
    "    chunk_overlap=200\n",
    ")\n",
    "splits = character_splitter.split_text(text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ef99e5c-2573-464c-ae54-c49da859513a",
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_documents = [Document(page_content=text, metadata={\"id\": str(i), \"source\": \"dense\"}) for i, text in enumerate(splits)]\n",
    "sparse_documents = [Document(page_content=text, metadata={\"id\": str(i), \"source\": \"sparse\"}) for i, text in enumerate(splits)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "6b13568c-d633-464d-8c43-0d55f34cc8c1",
   "metadata": {
    "executionInfo": {
     "elapsed": 9471,
     "status": "ok",
     "timestamp": 1715455766015,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "6b13568c-d633-464d-8c43-0d55f34cc8c1"
   },
   "outputs": [],
   "source": [
    "# Chroma Vector Store\n",
    "chroma_client = chromadb.Client()\n",
    "vectorstore = Chroma.from_documents(\n",
    "    documents=dense_documents,\n",
    "    embedding=embedding_function,\n",
    "    collection_name=collection_name,\n",
    "    client=chroma_client\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "177585c7-d346-43d3-9ea7-8777b41c06d2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# FAISS Vector Store\n",
    "from langchain_community.vectorstores import FAISS\n",
    "vectorstore = FAISS.from_documents(\n",
    "    documents=dense_documents,\n",
    "    embedding=embedding_function\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ca59dfe3-7557-444e-8af1-380244922107",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Weaviate Vector Store\n",
    "import weaviate\n",
    "from langchain_weaviate.vectorstores import WeaviateVectorStore\n",
    "from weaviate.embedded import EmbeddedOptions\n",
    "from langchain.vectorstores import Weaviate\n",
    "from tqdm import tqdm\n",
    "\n",
    "collection_name = \"Google_environmental_report\"\n",
    "\n",
    "weaviate_client = weaviate.Client(embedded_options=EmbeddedOptions())\n",
    "\n",
    "# Delete the existing 'google_environmental_report' class if it exists\n",
    "try:\n",
    "    weaviate_client.schema.delete_class(collection_name)\n",
    "except:\n",
    "    pass\n",
    "\n",
    "# Create the 'Google_environmental_report' class in the schema\n",
    "weaviate_client.schema.create_class({\n",
    "    \"class\": collection_name,\n",
    "    \"description\": \"Google Environmental Report\",\n",
    "    \"properties\": [\n",
    "        {\n",
    "            \"name\": \"text\",\n",
    "            \"dataType\": [\"text\"],\n",
    "            \"description\": \"Text content of the document\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"doc_id\",\n",
    "            \"dataType\": [\"string\"],\n",
    "            \"description\": \"Document ID\"\n",
    "        },\n",
    "        {\n",
    "            \"name\": \"source\",\n",
    "            \"dataType\": [\"string\"],\n",
    "            \"description\": \"Document source\"\n",
    "        }\n",
    "    ]\n",
    "})\n",
    "\n",
    "# Create new versions of these with \"doc_id\" because Weaviate reserves the \"id\" in the metadata for their internal id\n",
    "dense_documents = [Document(page_content=text, metadata={\"doc_id\": str(i), \"source\": \"dense\"}) for i, text in enumerate(splits)]\n",
    "sparse_documents = [Document(page_content=text, metadata={\"doc_id\": str(i), \"source\": \"sparse\"}) for i, text in enumerate(splits)]\n",
    "\n",
    "vectorstore = Weaviate(\n",
    "    client=weaviate_client,\n",
    "    embedding=embedding_function,\n",
    "    index_name=collection_name,\n",
    "    text_key=\"text\",\n",
    "    attributes=[\"doc_id\", \"source\"],\n",
    "    by_text=False\n",
    ")\n",
    "\n",
    "weaviate_client.batch.configure(batch_size=100)  # Configure batch\n",
    "\n",
    "# Batch option for loading data to Weaviate\n",
    "with weaviate_client.batch as batch:\n",
    "    for doc in tqdm(dense_documents, desc=\"Processing documents\"):\n",
    "        properties = {\n",
    "            \"text\": doc.page_content,\n",
    "            \"doc_id\": doc.metadata[\"doc_id\"],\n",
    "            \"source\": doc.metadata[\"source\"]\n",
    "        }\n",
    "        vector = embedding_function.embed_query(doc.page_content)\n",
    "        batch.add_data_object(\n",
    "            data_object=properties,\n",
    "            class_name=collection_name,\n",
    "            vector=vector\n",
    "        )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "749fcce7-3203-49e8-a62f-ecf2edce4570",
   "metadata": {},
   "outputs": [],
   "source": [
    "dense_retriever = vectorstore.as_retriever(search_kwargs={\"k\": 10})\n",
    "sparse_retriever = BM25Retriever.from_documents(sparse_documents, k=10)\n",
    "ensemble_retriever = EnsembleRetriever(retrievers=[dense_retriever, sparse_retriever], weights=[0.5, 0.5], c=0, k=10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6ce8df01-925b-45b5-8fb8-17b5c40c581f",
   "metadata": {
    "id": "6ce8df01-925b-45b5-8fb8-17b5c40c581f"
   },
   "outputs": [],
   "source": [
    "#### RETRIEVAL and GENERATION ####"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fac053d8-b871-4b50-b04e-28dec9fb3b0f",
   "metadata": {
    "executionInfo": {
     "elapsed": 355,
     "status": "ok",
     "timestamp": 1715455766356,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "fac053d8-b871-4b50-b04e-28dec9fb3b0f"
   },
   "outputs": [],
   "source": [
    "# Prompt\n",
    "prompt = hub.pull(\"jclemens24/rag-prompt\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ef30632-13dd-4a34-af33-cb8fab94f169",
   "metadata": {
    "executionInfo": {
     "elapsed": 3,
     "status": "ok",
     "timestamp": 1715455766356,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "5ef30632-13dd-4a34-af33-cb8fab94f169"
   },
   "outputs": [],
   "source": [
    "# Relevance check prompt\n",
    "relevance_prompt_template = PromptTemplate.from_template(\n",
    "    \"\"\"\n",
    "    Given the following question and retrieved context, determine if the context is relevant to the question.\n",
    "    Provide a score from 1 to 5, where 1 is not at all relevant and 5 is highly relevant.\n",
    "    Return ONLY the numeric score, without any additional text or explanation.\n",
    "\n",
    "    Question: {question}\n",
    "    Retrieved Context: {retrieved_context}\n",
    "\n",
    "    Relevance Score:\"\"\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8975479-b3e3-481d-ad7b-08b4eb3faaef",
   "metadata": {
    "executionInfo": {
     "elapsed": 2,
     "status": "ok",
     "timestamp": 1715455766356,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "e8975479-b3e3-481d-ad7b-08b4eb3faaef"
   },
   "outputs": [],
   "source": [
    "# Post-processing\n",
    "def format_docs(docs):\n",
    "    return \"\\n\\n\".join(doc.page_content for doc in docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd9db713-f705-4b65-800e-2c4e3d0e4ef4",
   "metadata": {
    "executionInfo": {
     "elapsed": 6,
     "status": "ok",
     "timestamp": 1715455766512,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "fd9db713-f705-4b65-800e-2c4e3d0e4ef4"
   },
   "outputs": [],
   "source": [
    "def extract_score(llm_output):\n",
    "    try:\n",
    "        score = float(llm_output.strip())\n",
    "        return score\n",
    "    except ValueError:\n",
    "        return 0\n",
    "\n",
    "def conditional_answer(x):\n",
    "    relevance_score = extract_score(x['relevance_score'])\n",
    "    if relevance_score < 4:\n",
    "        return \"I don't know.\"\n",
    "    else:\n",
    "        return x['answer']\n",
    "\n",
    "rag_chain_from_docs = (\n",
    "    RunnablePassthrough.assign(context=(lambda x: format_docs(x[\"context\"])))\n",
    "    | RunnableParallel(\n",
    "        {\"relevance_score\": (\n",
    "            RunnablePassthrough()\n",
    "            | (lambda x: relevance_prompt_template.format(question=x['question'], retrieved_context=x['context']))\n",
    "            | llm\n",
    "            | str_output_parser\n",
    "        ), \"answer\": (\n",
    "            RunnablePassthrough()\n",
    "            | prompt\n",
    "            | llm\n",
    "            | str_output_parser\n",
    "        )}\n",
    "    )\n",
    "    | RunnablePassthrough().assign(final_answer=conditional_answer)\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dc5c2ab0-9191-40f7-abf2-681f1c751429",
   "metadata": {
    "executionInfo": {
     "elapsed": 5,
     "status": "ok",
     "timestamp": 1715455766512,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "dc5c2ab0-9191-40f7-abf2-681f1c751429"
   },
   "outputs": [],
   "source": [
    "rag_chain_with_source = RunnableParallel(\n",
    "    {\"context\": ensemble_retriever, \"question\": RunnablePassthrough()}\n",
    ").assign(answer=rag_chain_from_docs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b30177a-f9ab-45e4-812d-33b0f97325bd",
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "executionInfo": {
     "elapsed": 3524,
     "status": "ok",
     "timestamp": 1715455770031,
     "user": {
      "displayName": "",
      "userId": ""
     },
     "user_tz": 240
    },
    "id": "8b30177a-f9ab-45e4-812d-33b0f97325bd",
    "outputId": "fb5f615f-202e-4409-f4c7-10aac3a577ee",
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# User Query\n",
    "result = rag_chain_with_source.invoke(user_query)\n",
    "retrieved_docs = result['context']\n",
    "\n",
    "print(f\"Original Question: {user_query}\\n\")\n",
    "print(f\"Relevance Score: {result['answer']['relevance_score']}\\n\")\n",
    "print(f\"Final Answer:\\n{result['answer']['final_answer']}\\n\\n\")\n",
    "print(\"Retrieved Documents:\")\n",
    "for i, doc in enumerate(retrieved_docs, start=1):\n",
    "    # note: if using the Weaviate vectorstore, change 'id' to 'doc_id'\n",
    "    print(f\"Document {i}: Document ID: {doc.metadata['doc_id']} source: {doc.metadata['source']}\")\n",
    "    print(f\"Content:\\n{doc.page_content}\\n\")"
   ]
  }
 ],
 "metadata": {
  "colab": {
   "name": "CHAPTER8-2_HYBRID-ENSEMBLE.ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
